from pyspark.sql import SparkSession
from pyspark.sql.functions import *
import matplotlib.pyplot as plt
import seaborn as sns
import csv
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.preprocessing import LabelEncoder
from datetime import datetime
import pandas as pd

plt.rcParams['font.sans-serif'] = ['SimHei']
# 读取CSV文件并转换为DataFrame
df = pd.read_csv('user_reviews.csv', parse_dates=['date'])
# 1. 数据概览
print("数据概览:")
print(df.head())
print("\n数据统计信息:")
print(df.describe())
print("\n缺失值检查:")
print(df.isnull().sum())
# 2. 商品类别平均评分（柱状图）
plt.figure(figsize=(12, 6))
category_avg = df.groupby('category')
['rating'].mean().sort_values(ascending=False)
sns.set_palette("viridis")
sns.barplot(x=category_avg.index, y=category_avg.values)
plt.title('各商品类别的平均评分')
plt.xlabel('商品类别')
plt.ylabel('平均评分')
plt.xticks(rotation=45)
plt.tight_layout()
plt.show()

# 3. 用户活跃度趋势（折线图）
plt.figure(figsize=(12, 6))
df['month'] = df['date'].dt.to_period('M').astype(str)
user_activity = df.groupby(['month', 'user_id']).size().unstack().fillna(0)
user_activity.plot(kind='line', figsize=(12, 6), marker='o')
plt.title('用户活跃度趋势')
plt.xlabel('月份')
plt.ylabel('评价次数')
plt.xticks(rotation=45)
plt.grid(True)
plt.tight_layout()
plt.show()
# 4. 评分分布（箱型图）
plt.figure(figsize=(10, 6))
sns.boxplot(x='category', y='rating', data=df, palette='Set2')
plt.title('各商品类别的评分分布')
plt.xlabel('商品类别')
plt.ylabel('评分')
plt.xticks(rotation=45)
plt.tight_layout()
plt.show()
# 5. 机器学习建模（评分预测）
# 数据预处理
le_user = LabelEncoder()
le_category = LabelEncoder()
df['user_encoded'] = le_user.fit_transform(df['user_id'])
df['category_encoded'] = le_category.fit_transform(df['category'])
df['day_of_week'] = df['date'].dt.dayofweek
df['month_num'] = df['date'].dt.month
# 特征和目标变量
X = df[['user_encoded', 'category_encoded', 'day_of_week', 'month_num']]
y = df['rating']
# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
random_state=42)
# 训练模型
model = RandomForestRegressor(n_estimators=100, random_state=42)
model.fit(X_train, y_train)
# 预测
y_pred = model.predict(X_test)
# 6. 模型评估
# 计算指标
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)
print(f"\n模型评估:")
print(f"均方误差(MSE): {mse:.2f}")
print(f"R²分数: {r2:.2f}")
# 可视化预测 vs 实际值
plt.figure(figsize=(10, 6))
plt.scatter(y_test, y_pred, alpha=0.6)
plt.plot([y.min(), y.max()], [y.min(), y.max()], 'k--', lw=2)
plt.xlabel('实际评分')
plt.ylabel('预测评分')
plt.title('预测评分 vs 实际评分')
plt.grid(True)
plt.tight_layout()
plt.show()
# 特征重要性
feature_importance = pd.DataFrame({
    'Feature': X.columns,
    'Importance': model.feature_importances_
}).sort_values('Importance', ascending=False)
plt.figure(figsize=(10, 6))
sns.barplot(x='Importance', y='Feature', data=feature_importance,palette='rocket')
plt.title('特征重要性')
plt.tight_layout()
plt.show()
